{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [],
            "source": [
                "import pandas as pd\n",
                "\n",
                "from sklearn.metrics import accuracy_score, cohen_kappa_score, confusion_matrix"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>id</th>\n",
                            "      <th>text</th>\n",
                            "      <th>sentiment_C1</th>\n",
                            "      <th>stance_C1</th>\n",
                            "      <th>sentiment_C2</th>\n",
                            "      <th>stance_C2</th>\n",
                            "      <th>target</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>wm_0</td>\n",
                            "      <td>Target: Women's March\\n\\nLondon Women's March ...</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>1</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>wm</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>wm_1</td>\n",
                            "      <td>Target: Women's March\\n\\nRip to those that act...</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>0</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>wm</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>wm_2</td>\n",
                            "      <td>Target: Women's March\\n\\nLook how much I do ca...</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>0</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>wm</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>wm_3</td>\n",
                            "      <td>Target: Women's March\\n\\nOh no this will defin...</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>0</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>0.0</td>\n",
                            "      <td>wm</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>wm_4</td>\n",
                            "      <td>Target: Women's March\\n\\nThe women's marches w...</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>1</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>wm</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "     id                                               text  sentiment_C1  \\\n",
                            "0  wm_0  Target: Women's March\\n\\nLondon Women's March ...           1.0   \n",
                            "1  wm_1  Target: Women's March\\n\\nRip to those that act...           0.0   \n",
                            "2  wm_2  Target: Women's March\\n\\nLook how much I do ca...           1.0   \n",
                            "3  wm_3  Target: Women's March\\n\\nOh no this will defin...           0.0   \n",
                            "4  wm_4  Target: Women's March\\n\\nThe women's marches w...           1.0   \n",
                            "\n",
                            "   stance_C1  sentiment_C2  stance_C2 target  \n",
                            "0          1           1.0        1.0     wm  \n",
                            "1          0           0.0        0.0     wm  \n",
                            "2          0           1.0        0.0     wm  \n",
                            "3          0           0.0        0.0     wm  \n",
                            "4          1           1.0        1.0     wm  "
                        ]
                    },
                    "execution_count": 2,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "IRR_df = pd.read_csv('./../data/intercoder_reliability_check.csv')\n",
                "\n",
                "IRR_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [],
            "source": [
                "%%capture output\n",
                "\n",
                "#TABLE A1\n",
                "\n",
                "print(len(IRR_df[IRR_df['target']=='wm']))\n",
                "\n",
                "print(accuracy_score(IRR_df['sentiment_C1'][IRR_df['target']=='wm'], IRR_df['sentiment_C2'][IRR_df['target']=='wm']))\n",
                "print(cohen_kappa_score(IRR_df['sentiment_C1'][IRR_df['target']=='wm'], IRR_df['sentiment_C2'][IRR_df['target']=='wm']))\n",
                "\n",
                "print(confusion_matrix(IRR_df['sentiment_C1'][IRR_df['target']=='wm'], IRR_df['sentiment_C2'][IRR_df['target']=='wm']))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "211\n",
                        "0.8767772511848341\n",
                        "0.753548966756514\n",
                        "[[93 13]\n",
                        " [13 92]]\n"
                    ]
                }
            ],
            "source": [
                "output()\n",
                "\n",
                "with open('./../tables/Table_A1.txt', 'w') as f:\n",
                "    f.write(output.stdout)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [],
            "source": [
                "%%capture output\n",
                "\n",
                "#TABLE A2\n",
                "\n",
                "print(len(IRR_df[IRR_df['target']=='wm']))\n",
                "\n",
                "print(accuracy_score(IRR_df['stance_C1'][IRR_df['target']=='wm'], IRR_df['stance_C2'][IRR_df['target']=='wm']))\n",
                "print(cohen_kappa_score(IRR_df['stance_C1'][IRR_df['target']=='wm'], IRR_df['stance_C2'][IRR_df['target']=='wm']))\n",
                "\n",
                "print(confusion_matrix(IRR_df['stance_C1'][IRR_df['target']=='wm'], IRR_df['stance_C2'][IRR_df['target']=='wm']))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "211\n",
                        "0.9146919431279621\n",
                        "0.8290882908829088\n",
                        "[[ 92  10]\n",
                        " [  8 101]]\n"
                    ]
                }
            ],
            "source": [
                "output()\n",
                "\n",
                "with open('./../tables/Table_A2.txt', 'w') as f:\n",
                "    f.write(output.stdout)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [],
            "source": [
                "%%capture output\n",
                "\n",
                "#TABLE A3\n",
                "\n",
                "print(len(IRR_df[IRR_df['target']=='kav']))\n",
                "\n",
                "print(accuracy_score(IRR_df['sentiment_C1'][IRR_df['target']=='kav'], IRR_df['sentiment_C2'][IRR_df['target']=='kav']))\n",
                "\n",
                "print(cohen_kappa_score(IRR_df['sentiment_C1'][IRR_df['target']=='kav'], IRR_df['sentiment_C2'][IRR_df['target']=='kav']))\n",
                "\n",
                "\n",
                "print(confusion_matrix(IRR_df['sentiment_C1'][IRR_df['target']=='kav'], IRR_df['sentiment_C2'][IRR_df['target']=='kav']))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "200\n",
                        "0.81\n",
                        "0.3041567478483794\n",
                        "[[151   0]\n",
                        " [ 38  11]]\n"
                    ]
                }
            ],
            "source": [
                "output()\n",
                "\n",
                "with open('./../tables/Table_A3.txt', 'w') as f:\n",
                "    f.write(output.stdout)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [],
            "source": [
                "%%capture output\n",
                "\n",
                "#TABLE A4\n",
                "\n",
                "print(len(IRR_df[IRR_df['target']=='kav']))\n",
                "\n",
                "print(accuracy_score(IRR_df['stance_C1'][IRR_df['target']=='kav'], IRR_df['stance_C2'][IRR_df['target']=='kav']))\n",
                "print(cohen_kappa_score(IRR_df['stance_C1'][IRR_df['target']=='kav'], IRR_df['stance_C2'][IRR_df['target']=='kav']))\n",
                "\n",
                "print(confusion_matrix(IRR_df['stance_C1'][IRR_df['target']=='kav'], IRR_df['stance_C2'][IRR_df['target']=='kav']))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "200\n",
                        "0.975\n",
                        "0.9499399279134962\n",
                        "[[101   2]\n",
                        " [  3  94]]\n"
                    ]
                }
            ],
            "source": [
                "output()\n",
                "\n",
                "with open('./../tables/Table_A4.txt', 'w') as f:\n",
                "    f.write(output.stdout)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "interpreter": {
            "hash": "b4a597a0c75f172e157fd825f25a2bad8e418bebf6f03bd8f8b243293e63e4ba"
        },
        "kernelspec": {
            "display_name": "Python 3.8.5 64-bit ('base': conda)",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.7.6"
        },
        "orig_nbformat": 4
    },
    "nbformat": 4,
    "nbformat_minor": 2
}
